import random
import time
import os
import tkinter as tk
from tkinter import messagebox, font

class FrozenLake:
    def __init__(self, size=4, slip_probability=0.3):
        """
        初始化冰湖游戏
        size: 网格大小（默认4x4）
        slip_probability: 冰面打滑的概率（默认0.3）
        """
        self.size = size
        self.slip_probability = slip_probability
        self.reset()
        self._create_environment()
        self.gui = None  # GUI引用
        
    def _create_environment(self):
        """
        创建游戏环境
        S: 起点 (safe)
        F: 结冰的湖面 (frozen, safe)
        H: 洞 (hole, terminal state, bad)
        G: 目标 (goal, terminal state, good)
        """
        # 预设4x4的经典冰湖环境布局
        if self.size == 4:
            self.grid = [
                ['S', 'F', 'F', 'F'],
                ['F', 'H', 'F', 'H'],
                ['F', 'F', 'F', 'H'],
                ['H', 'F', 'F', 'G']
            ]
        else:
            # 对于其他大小，创建一个随机的冰湖环境
            self.grid = []
            for i in range(self.size):
                row = []
                for j in range(self.size):
                    # 确保起点和终点位置
                    if i == 0 and j == 0:
                        row.append('S')
                    elif i == self.size - 1 and j == self.size - 1:
                        row.append('G')
                    else:
                        # 随机生成洞和冰面，洞的概率为0.2
                        row.append('H' if random.random() < 0.2 else 'F')
                self.grid.append(row)
    
    def reset(self):
        """
        重置游戏状态
        """
        self.current_pos = [0, 0]  # 起点位置
        self.done = False
        self.reward = 0
        self.history = []  # 记录移动历史
        return self.current_pos
    
    def get_state(self):
        """
        获取当前状态
        """
        return self.current_pos
    
    def is_terminal(self):
        """
        检查是否达到终止状态（掉进洞或到达终点）
        """
        return self.done
    
    def step(self, action):
        """
        执行一步动作
        action: 0=上, 1=右, 2=下, 3=左
        返回: (新位置, 奖励, 是否结束, 信息)
        """
        # 记录原始位置和选择的动作
        original_pos = self.current_pos.copy()
        action_name = ['上', '右', '下', '左'][action]
        
        # 考虑冰面打滑的情况
        if random.random() < self.slip_probability:
            # 打滑时，随机选择一个方向（非原方向）
            possible_actions = [a for a in range(4) if a != action]
            actual_action = random.choice(possible_actions)
            slip_info = f"(打滑! 方向变为{['上', '右', '下', '左'][actual_action]})".rjust(20)
        else:
            actual_action = action
            slip_info = "(正常移动)".rjust(20)
        
        # 计算新位置
        new_pos = self.current_pos.copy()
        if actual_action == 0:  # 上
            new_pos[0] = max(0, new_pos[0] - 1)
        elif actual_action == 1:  # 右
            new_pos[1] = min(self.size - 1, new_pos[1] + 1)
        elif actual_action == 2:  # 下
            new_pos[0] = min(self.size - 1, new_pos[0] + 1)
        elif actual_action == 3:  # 左
            new_pos[1] = max(0, new_pos[1] - 1)
        
        # 更新当前位置
        self.current_pos = new_pos
        
        # 检查新位置的状态
        pos_type = self.grid[new_pos[0]][new_pos[1]]
        if pos_type == 'H':  # 掉进洞
            self.done = True
            self.reward = -1
            result = "掉进洞! 游戏结束."
        elif pos_type == 'G':  # 到达终点
            self.done = True
            self.reward = 1
            result = "到达终点! 游戏胜利!"
        else:  # 继续游戏
            self.reward = 0
            result = "继续前进"
        
        # 记录这一步的历史
        self.history.append({
            'original_pos': original_pos,
            'chosen_action': action,
            'actual_action': actual_action,
            'new_pos': new_pos,
            'slip_info': slip_info,
            'result': result
        })
        
        return new_pos, self.reward, self.done, {'result': result}
    
    def print_environment(self):
        """
        打印当前环境状态
        """
        print("-" * (self.size * 4 + 1))
        for i in range(self.size):
            row = "|"
            for j in range(self.size):
                if self.current_pos == [i, j]:
                    row += " A |"  # A表示Agent的位置
                else:
                    row += f" {self.grid[i][j]} |"
            print(row)
            print("-" * (self.size * 4 + 1))
    
    def print_step_info(self, step_num):
        """
        打印单步移动信息
        """
        if step_num < len(self.history):
            step = self.history[step_num]
            print(f"\n步骤 {step_num + 1}:")
            print(f"起始位置: ({step['original_pos'][0]}, {step['original_pos'][1]})")
            print(f"选择动作: {['上', '右', '下', '左'][step['chosen_action']]} {step['slip_info']}")
            print(f"实际位置: ({step['new_pos'][0]}, {step['new_pos'][1]})")
            print(f"结果: {step['result']}")
    
    def print_history(self):
        """
        打印完整的移动历史
        """
        print("\n游戏历史:")
        print("-" * 50)
        for i, step in enumerate(self.history):
            print(f"步骤 {i + 1}:")
            print(f"  起始位置: ({step['original_pos'][0]}, {step['original_pos'][1]})")
            print(f"  选择动作: {['上', '右', '下', '左'][step['chosen_action']]} {step['slip_info']}")
            print(f"  实际位置: ({step['new_pos'][0]}, {step['new_pos'][1]})")
            print(f"  结果: {step['result']}")
            print("-" * 50)

class RandomAgent:
    """
    随机选择动作的Agent
    """
    def __init__(self):
        self.name = "随机探索者"
    
    def choose_action(self, state):
        """
        随机选择一个动作
        """
        return random.choice([0, 1, 2, 3])

class RuleBasedAgent:
    """
    基于规则的Agent，尝试向目标方向移动
    """
    def __init__(self, goal_pos):
        self.name = "规则导向者"
        self.goal_pos = goal_pos
    
    def choose_action(self, env):
        """
        基于当前位置和目标位置的规则选择动作，增加避洞逻辑
        现在接收环境对象而非直接的状态
        """
        # 从环境对象获取当前状态（位置）
        state = env.get_state()
        
        # 获取网格信息以检查洞的位置
        grid = env.grid
        size = env.size
        
        # 计算各个方向移动后的新位置
        def get_new_pos(action):
            new_pos = state.copy()
            if action == 0:  # 上
                new_pos[0] -= 1
            elif action == 1:  # 右
                new_pos[1] += 1
            elif action == 2:  # 下
                new_pos[0] += 1
            elif action == 3:  # 左
                new_pos[1] -= 1
            return new_pos
        
        # 检查位置是否有效且安全（不是洞）
        def is_safe_pos(pos):
            # 检查是否在网格范围内
            if 0 <= pos[0] < size and 0 <= pos[1] < size:
                # 检查是否不是洞
                return grid[pos[0]][pos[1]] != 'H'
            return False
        
        # 获取所有安全的动作
        safe_actions = []
        for action in [0, 1, 2, 3]:
            new_pos = get_new_pos(action)
            if is_safe_pos(new_pos):
                safe_actions.append(action)
        
        # 如果有安全动作
        if safe_actions:
            # 优先选择朝向目标的安全动作
            # 下方向和右方向的优先级
            preferred_actions = []
            if state[0] < self.goal_pos[0] and 2 in safe_actions:
                preferred_actions.append(2)  # 下
            if state[1] < self.goal_pos[1] and 1 in safe_actions:
                preferred_actions.append(1)  # 右
            
            # 如果有朝向目标的安全动作，选择其中一个
            if preferred_actions:
                return random.choice(preferred_actions)
            
            # 否则从所有安全动作中随机选择
            return random.choice(safe_actions)
        
        # 如果没有安全动作（被洞包围），随机选择一个动作
        return random.choice([0, 1, 2, 3])

class QLearningAgent:
    """
    基于表格的Q-learning强化学习Agent
    """
    def __init__(self, env_size=4, learning_rate=0.1, discount_factor=0.9, epsilon=0.1):
        self.name = "Q-learning探索者"
        self.env_size = env_size
        self.learning_rate = learning_rate  # 学习率
        self.discount_factor = discount_factor  # 折扣因子
        self.epsilon = epsilon  # 探索率
        
        # 初始化Q表格，形状为[size, size, 4]，表示每个位置的四个动作的Q值
        self.q_table = {}
        for i in range(env_size):
            for j in range(env_size):
                # 为每个状态位置的四个动作初始化为0
                self.q_table[(i, j)] = [0.0, 0.0, 0.0, 0.0]
        
        # 记录上一个状态和动作，用于学习
        self.prev_state = None
        self.prev_action = None
    
    def choose_action(self, env):
        """
        使用ε-贪婪策略选择动作
        """
        # 获取当前状态
        current_state = env.get_state()
        state_tuple = (current_state[0], current_state[1])
        
        # ε-贪婪策略：以ε的概率随机探索，否则选择Q值最大的动作
        if random.random() < self.epsilon:
            # 随机选择动作
            action = random.choice([0, 1, 2, 3])
        else:
            # 选择Q值最大的动作，如果有多个相同最大值，随机选择一个
            q_values = self.q_table[state_tuple]
            max_q = max(q_values)
            best_actions = [i for i, q in enumerate(q_values) if q == max_q]
            action = random.choice(best_actions)
        
        # 记录当前状态和动作
        self.prev_state = state_tuple
        self.prev_action = action
        
        return action
    
    def learn(self, state, reward, done):
        """
        根据Q-learning算法更新Q表格
        Q(s,a) = Q(s,a) + α[r + γ * max_a' Q(s',a') - Q(s,a)]
        """
        if self.prev_state is not None and self.prev_action is not None:
            # 转换当前状态为元组
            state_tuple = (state[0], state[1])
            
            # 获取当前Q值
            current_q = self.q_table[self.prev_state][self.prev_action]
            
            # 计算下一状态的最大Q值
            if done:
                # 如果是终止状态，max_q_next为0
                max_q_next = 0
            else:
                max_q_next = max(self.q_table[state_tuple])
            
            # 更新Q值
            new_q = current_q + self.learning_rate * (reward + self.discount_factor * max_q_next - current_q)
            self.q_table[self.prev_state][self.prev_action] = new_q
        
        # 如果是终止状态，重置历史记录
        if done:
            self.prev_state = None
            self.prev_action = None


class FrozenLakeGUI:
    """
    冰湖游戏的图形用户界面
    """
    def __init__(self, master):
        self.master = master
        self.env = None
        self.agent = None
        self.q_learning_agent = None  # 单独保存Q-learning Agent实例以保持学习状态
        self.agent_type = tk.StringVar(value="rule_based")  # 默认选择规则导向者
        
        self.master.title("冰湖游戏 (Frozen Lake)")
        self.master.geometry("1000x800")
        self.master.resizable(True, True)
        
        # 设置中文字体
        try:
            self.font = font.Font(family="SimHei", size=12)
            self.title_font = font.Font(family="SimHei", size=16, weight="bold")
            self.grid_font = font.Font(family="SimHei", size=14, weight="bold")
        except:
            # 备选字体
            self.font = font.Font(family="Arial", size=12)
            self.title_font = font.Font(family="Arial", size=16, weight="bold")
            self.grid_font = font.Font(family="Arial", size=14, weight="bold")
        
        # 创建标题
        self.title_frame = tk.Frame(master)
        self.title_frame.pack(pady=10, fill=tk.X)
        
        self.title_label = tk.Label(self.title_frame, text="冰湖游戏 (Frozen Lake)", font=self.title_font)
        self.title_label.pack()
        
        # 添加Agent选择区域
        self.agent_select_frame = tk.Frame(master)
        self.agent_select_frame.pack(pady=10, fill=tk.X)
        
        tk.Label(self.agent_select_frame, text="选择Agent类型:", font=self.font).pack(side=tk.LEFT, padx=10)
        
        tk.Radiobutton(self.agent_select_frame, text="随机探索者", variable=self.agent_type, value="random", 
                      font=self.font).pack(side=tk.LEFT, padx=10)
        tk.Radiobutton(self.agent_select_frame, text="规则导向者", variable=self.agent_type, value="rule_based", 
                      font=self.font).pack(side=tk.LEFT, padx=10)
        tk.Radiobutton(self.agent_select_frame, text="Q-learning探索者", variable=self.agent_type, value="q_learning", 
                      font=self.font).pack(side=tk.LEFT, padx=10)
        
        self.agent_label = tk.Label(self.agent_select_frame, text="使用Agent: ", font=self.font)
        self.agent_label.pack(side=tk.LEFT, padx=10)
        
        # 创建信息显示区域
        self.info_frame = tk.Frame(master)
        self.info_frame.pack(pady=10, fill=tk.X)
        
        self.step_label = tk.Label(self.info_frame, text="步骤: 0", font=self.font)
        self.step_label.pack(anchor="w", padx=10)
        
        self.action_label = tk.Label(self.info_frame, text="", font=self.font)
        self.action_label.pack(anchor="w", padx=10)
        
        self.result_label = tk.Label(self.info_frame, text="", font=self.font)
        self.result_label.pack(anchor="w", padx=10)
        
        # 创建游戏步骤显示区域
        self.steps_frame = tk.Frame(master)
        self.steps_frame.pack(pady=10, fill=tk.BOTH, expand=True)
        
        tk.Label(self.steps_frame, text="游戏步骤记录:", font=self.font).pack(anchor="w", padx=10, pady=5)
        
        # 创建滚动条和文本框来显示步骤
        self.steps_scrollbar = tk.Scrollbar(self.steps_frame)
        self.steps_scrollbar.pack(side=tk.RIGHT, fill=tk.Y)
        
        self.steps_text = tk.Text(self.steps_frame, height=8, width=70, font=self.font, wrap=tk.WORD,
                                 yscrollcommand=self.steps_scrollbar.set)
        self.steps_text.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=10)
        self.steps_scrollbar.config(command=self.steps_text.yview)
        
        # 创建主要内容区域（左右布局）
        self.main_content_frame = tk.Frame(master)
        self.main_content_frame.pack(pady=10, fill=tk.BOTH, expand=True)
        
        # 创建左侧Q表格显示区域
        self.q_table_frame = tk.Frame(self.main_content_frame)
        self.q_table_frame.pack(side=tk.LEFT, padx=10, fill=tk.Y)
        
        self.q_table_label = tk.Label(self.q_table_frame, text="Q表格 (上-右-下-左):", font=self.font)
        self.q_table_label.pack(anchor="w", padx=10, pady=5)
        
        # 创建滚动条和文本框来显示Q表格
        self.q_table_scrollbar = tk.Scrollbar(self.q_table_frame)
        self.q_table_scrollbar.pack(side=tk.RIGHT, fill=tk.Y)
        
        self.q_table_text = tk.Text(self.q_table_frame, height=20, width=50, font=font.Font(family="Courier", size=10),
                                 wrap=tk.NONE, yscrollcommand=self.q_table_scrollbar.set)
        self.q_table_text.pack(side=tk.LEFT, fill=tk.BOTH, expand=True, padx=10)
        self.q_table_scrollbar.config(command=self.q_table_text.yview)
        
        # 创建右侧游戏区域
        self.game_area_frame = tk.Frame(self.main_content_frame)
        self.game_area_frame.pack(side=tk.RIGHT, padx=10, fill=tk.BOTH, expand=True)
        
        # 创建网格显示区域
        self.grid_frame = tk.Frame(self.game_area_frame)
        self.grid_frame.pack(pady=10)
        
        # 初始占位，后续在start_game中创建网格
        self.cells = []
        
        # 初始化Q表格显示
        self.update_q_table_display()
        
        # 创建控制按钮区域 - 水平排列按钮
        self.control_frame = tk.Frame(master)
        self.control_frame.pack(pady=10, fill=tk.X, expand=False)
        
        # 按钮水平排列在一行中
        self.start_button = tk.Button(self.control_frame, text="开始游戏", command=self.start_game, font=self.font,
                                     width=15, height=2)
        self.start_button.pack(side=tk.LEFT, padx=10, pady=5)
        
        self.step_button = tk.Button(self.control_frame, text="单步执行", command=self.step_game, font=self.font,
                                    width=15, height=2, state=tk.DISABLED)
        self.step_button.pack(side=tk.LEFT, padx=10, pady=5)
        
        self.auto_button = tk.Button(self.control_frame, text="自动运行", command=self.toggle_auto_run, font=self.font,
                                    width=15, height=2, state=tk.DISABLED)
        self.auto_button.pack(side=tk.LEFT, padx=10, pady=5)
        
        # 自动运行相关变量
        self.auto_running = False
        self.delay = 1000  # 毫秒
        self.step_num = 0
        self.steps_history = []
        
        # 添加额外的提示标签
        self.tip_label = tk.Label(master, text="请选择Agent类型并点击'开始游戏'按钮开始", font=self.font, fg="blue")
        self.tip_label.pack(pady=10)
    
    def update_grid(self):
        """
        更新网格显示
        """
        # 状态颜色映射
        colors = {
            'S': '#90EE90',  # 浅绿色 - 起点
            'F': '#87CEEB',  # 浅蓝色 - 冰面
            'H': '#A9A9A9',  # 灰色 - 洞
            'G': '#FFD700'   # 金色 - 终点
        }
        
        for i in range(self.env.size):
            for j in range(self.env.size):
                cell_frame, cell_label = self.cells[i][j]
                cell_type = self.env.grid[i][j]
                
                # 设置单元格颜色
                if self.env.current_pos == [i, j]:
                    # 智能体所在位置
                    cell_frame.config(bg='#FF6347')  # 红色
                    cell_label.config(text='A', bg='#FF6347', fg='white')
                else:
                    # 普通位置
                    cell_frame.config(bg=colors.get(cell_type, '#FFFFFF'))
                    cell_label.config(text=cell_type, bg=colors.get(cell_type, '#FFFFFF'), 
                                     fg='black' if cell_type != 'G' else '#8B0000')
    
    def update_q_table_display(self):
        """
        更新Q表格显示
        """
        # 清空现有内容
        self.q_table_text.delete(1.0, tk.END)
        
        # 检查是否使用的是Q-learning Agent
        if hasattr(self.agent, 'q_table'):
            q_table = self.agent.q_table
            env_size = self.agent.env_size
            
            # 显示表头
            self.q_table_text.insert(tk.END, "位置      | 上     右     下     左\n")
            self.q_table_text.insert(tk.END, "-" * 50 + "\n")
            
            # 按行显示Q表格值
            for i in range(env_size):
                for j in range(env_size):
                    state = (i, j)
                    q_values = q_table[state]
                    # 格式化显示每个动作的Q值，保留3位小数
                    q_str = f"({i},{j})  | {q_values[0]:.3f}  {q_values[1]:.3f}  {q_values[2]:.3f}  {q_values[3]:.3f}\n"
                    self.q_table_text.insert(tk.END, q_str)
        else:
            # 如果不是Q-learning Agent，显示提示信息
            self.q_table_text.insert(tk.END, "当前Agent不是Q-learning探索者，无Q表格数据")
    
    def update_info(self):
        """
        更新信息显示
        """
        self.step_label.config(text=f"步骤: {self.step_num}")
        
        if self.step_num > 0 and self.step_num <= len(self.env.history):
            step = self.env.history[self.step_num - 1]
            action_names = ['上', '右', '下', '左']
            action_text = f"选择动作: {action_names[step['chosen_action']]} {step['slip_info']}"
            self.action_label.config(text=action_text)
            self.result_label.config(text=f"结果: {step['result']}")
            
            # 如果是Q-learning Agent，更新Q表格显示
            if hasattr(self.agent, 'q_table'):
                self.update_q_table_display()
    
    def start_game(self):
        """
        开始游戏
        """
        # 获取选择的Agent类型
        agent_type = self.agent_type.get()
        
        # 创建环境
        self.env = FrozenLake(size=4)
        self.env.gui = self
        
        # 根据选择创建Agent
        if agent_type == "random":
            self.agent = RandomAgent()
        elif agent_type == "q_learning":
            # 如果是第一次创建Q-learning Agent或者切换回Q-learning
            if self.q_learning_agent is None or not isinstance(self.agent, QLearningAgent):
                self.q_learning_agent = QLearningAgent(env_size=4)
            self.agent = self.q_learning_agent
        else:
            self.agent = RuleBasedAgent(goal_pos=[3, 3])
        
        # 更新Agent标签
        self.agent_label.config(text=f"使用Agent: {self.agent.name}")
        
        # 重置环境和Agent位置
        self.env.reset()
        self.step_num = 0
        self.steps_history = []
        self.steps_text.delete(1.0, tk.END)  # 清空步骤文本框
        
        # 创建或更新网格
        if not self.cells:
            self.create_grid()
        self.update_grid()
        
        # 更新显示
        self.step_label.config(text="步骤: 0")
        self.action_label.config(text="")
        self.result_label.config(text="")
        
        # 更新Q表格显示
        self.update_q_table_display()
        
        # 记录初始状态
        start_pos = self.env.current_pos
        self.add_step(f"初始位置: 行 {start_pos[0]}, 列 {start_pos[1]}")
        
        # 启用控制按钮
        self.start_button.config(state=tk.DISABLED)
        self.step_button.config(state=tk.NORMAL)
        self.auto_button.config(state=tk.NORMAL)
        self.tip_label.config(text="游戏已开始")
    
    def create_grid(self):
        """
        创建网格单元格
        """
        # 清空现有的网格
        for widget in self.grid_frame.winfo_children():
            widget.destroy()
        
        self.cell_size = max(60, 500 // self.env.size)  # 自适应单元格大小
        self.cells = []
        
        # 创建网格单元格
        for i in range(self.env.size):
            row = []
            for j in range(self.env.size):
                cell_frame = tk.Frame(self.grid_frame, width=self.cell_size, height=self.cell_size,
                                     highlightthickness=2, highlightbackground="black")
                cell_frame.grid(row=i, column=j, padx=2, pady=2)
                cell_frame.grid_propagate(False)  # 固定大小
                
                cell_label = tk.Label(cell_frame, text="", font=self.grid_font)
                cell_label.place(relx=0.5, rely=0.5, anchor="center")
                
                row.append((cell_frame, cell_label))
            self.cells.append(row)
    
    def add_step(self, step_text):
        """
        添加步骤记录
        """
        self.steps_history.append(step_text)
        self.steps_text.insert(tk.END, f"{len(self.steps_history)}. {step_text}\n")
        self.steps_text.see(tk.END)  # 滚动到最后一行
    
    def step_game(self):
        """
        单步执行游戏
        """
        if not self.env.is_terminal():
            # Agent选择动作
            action = self.agent.choose_action(self.env)
            action_name = ['上', '右', '下', '左'][action]
            
            # 记录前一位置
            prev_pos = self.env.current_pos.copy()
            
            # 执行动作
            new_pos, reward, done, info = self.env.step(action)
            
            # 如果是Q-learning Agent，进行学习
            if hasattr(self.agent, 'learn'):
                self.agent.learn(new_pos, reward, done)
                # 更新Q表格显示
                self.update_q_table_display()
            
            # 更新步骤计数
            self.step_num += 1
            
            # 更新显示
            self.update_grid()
            self.step_label.config(text=f"步骤: {self.step_num}")
            
            # 获取打滑信息
            step_history = self.env.history[-1] if self.env.history else {}
            chosen_action = step_history.get('chosen_action', action)
            actual_action = step_history.get('actual_action', action)
            slip_info = step_history.get('slip_info', "")
            
            # 检查是否发生了滑动
            is_slip = chosen_action != actual_action
            slip_status = "(发生滑动! 方向变为" + ['上', '右', '下', '左'][actual_action] + ")" if is_slip else "(未发生滑动)"
            
            self.action_label.config(text=f"选择动作: {action_name} {slip_status} | 从({prev_pos[0]},{prev_pos[1]})到({new_pos[0]},{new_pos[1]})")
            
            # 记录步骤，包含滑动信息
            step_text = f"选择动作: {action_name} {slip_status} | 从({prev_pos[0]},{prev_pos[1]})移动到({new_pos[0]},{new_pos[1]})"
            self.add_step(step_text)
            
            # 检查游戏状态
            if self.env.is_terminal():
                if self.env.reward == 1:
                    result = "成功到达终点！"
                    self.result_label.config(text=result, fg="green")
                    self.add_step("游戏结果: 成功到达终点！")
                else:
                    result = "掉进洞里，游戏失败！"
                    self.result_label.config(text=result, fg="red")
                    self.add_step("游戏结果: 掉进洞里，游戏失败！")
                
                # 禁用控制按钮
                self.step_button.config(state=tk.DISABLED)
                self.auto_button.config(state=tk.DISABLED)
                self.start_button.config(state=tk.NORMAL)
                self.tip_label.config(text="游戏结束，请点击'开始游戏'重新开始")
    
    def toggle_auto_run(self):
        """
        切换自动运行状态
        """
        if self.auto_running:
            self.auto_running = False
            self.auto_button.config(text="自动运行")
            self.step_button.config(state=tk.NORMAL)
        else:
            self.auto_running = True
            self.auto_button.config(text="停止")
            self.step_button.config(state=tk.DISABLED)
            # 设置自动运行的次数为15
            self.auto_runs_remaining = 15
            self.auto_step()
    
    def auto_step(self):
        """
        自动执行下一步
        """
        if self.auto_running and self.auto_runs_remaining > 0:
            if self.env.is_terminal():
                # 游戏结束后重置并开始新回合
                # 重置环境但保留当前的Agent
                self.env.reset()
                self.step_num = 0
                self.steps_text.delete(1.0, tk.END)
                self.update_grid()
                self.step_label.config(text="步骤: 0")
                self.action_label.config(text="")
                self.result_label.config(text="")
                start_pos = self.env.current_pos
                self.add_step(f"初始位置: 行 {start_pos[0]}, 列 {start_pos[1]}")
                self.auto_runs_remaining -= 1
                self.tip_label.config(text=f"自动运行中: {15-self.auto_runs_remaining}/15 回合")
            else:
                self.step_game()
            # 设置定时器，继续自动运行
            self.master.after(self.delay, self.auto_step)
        else:
            # 达到15次或被手动停止
            self.auto_running = False
            self.auto_button.config(text="自动运行")
            self.step_button.config(state=tk.NORMAL)
            self.tip_label.config(text="自动运行已完成15次游戏")
            # 显示最终的Q表格
            if hasattr(self.agent, 'q_table'):
                self.update_q_table_display()


def clear_terminal():
    """
    清空终端显示
    """
    # 根据操作系统类型选择清空命令
    os.system('cls' if os.name == 'nt' else 'clear')

def play_game(env, agent, delay=1.0):
    """
    运行游戏并显示每一步
    """
    clear_terminal()  # 游戏开始前清空终端
    print(f"\n===== 冰湖游戏开始 =====")
    print(f"使用Agent: {agent.name}")
    env.reset()
    
    step_num = 0
    while not env.is_terminal():
        # 显示当前环境
        print(f"\n步骤 {step_num + 1}:")
        env.print_environment()
        
        # Agent选择动作
        action = agent.choose_action(env)
        
        # 执行动作
        new_state, reward, done, info = env.step(action)
        
        # 显示移动信息
        env.print_step_info(step_num)
        
        step_num += 1
        time.sleep(delay)  # 延迟以便观察
    
    # 显示最终状态
    print(f"\n最终状态:")
    env.print_environment()
    
    # 显示完整历史
    env.print_history()
    
    print(f"\n游戏结束! 总共进行了 {step_num} 步.")

def play_game_gui(env, agent):
    """
    在GUI窗口中运行游戏
    """
    root = tk.Tk()
    app = FrozenLakeGUI(root, env, agent)
    
    # 初始更新网格显示
    app.update_grid()
    
    # 启动GUI主循环
    root.mainloop()


def main():
    # 创建GUI
    root = tk.Tk()
    gui = FrozenLakeGUI(root)
    
    # 启动主循环
    root.mainloop()


if __name__ == "__main__":
    main()